{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "de53995b-32ed-4722-8cac-ba104c8efacb",
   "metadata": {},
   "source": [
    "# 导入环境"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "52fac949-4150-4091-b0c3-2968ab5e385c",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/miniconda3/envs/py310/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "from datasets import Dataset\n",
    "import pandas as pd\n",
    "from transformers import AutoTokenizer, AutoModelForCausalLM, DataCollatorForSeq2Seq, TrainingArguments, Trainer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "e098d9eb",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# 将JSON文件转换为CSV文件\n",
    "df = pd.read_json('./huanhuan.json') # 注意修改\n",
    "ds = Dataset.from_pandas(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "8ac92d42-efae-49b1-a00e-ccaa75b98938",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'instruction': ['小姐，别的秀女都在求中选，唯有咱们小姐想被撂牌子，菩萨一定记得真真儿的——',\n",
       "  '这个温太医啊，也是古怪，谁不知太医不得皇命不能为皇族以外的人请脉诊病，他倒好，十天半月便往咱们府里跑。',\n",
       "  '嬛妹妹，刚刚我去府上请脉，听甄伯母说你来这里进香了。'],\n",
       " 'input': ['', '', ''],\n",
       " 'output': ['嘘——都说许愿说破是不灵的。', '你们俩话太多了，我该和温太医要一剂药，好好治治你们。', '出来走走，也是散心。']}"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ds[:3]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "51d05e5d-d14e-4f03-92be-9a9677d41918",
   "metadata": {},
   "source": [
    "# 处理数据集"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "74ee5a67-2e55-4974-b90e-cbf492de500a",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "tokenizer = AutoTokenizer.from_pretrained('/model/ModelScope/ZhipuAI/GLM-4.5-Air')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "714f9eab",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[gMASK]<sop><|system|>\n",
      "You are a helpful assistant.<|user|>\n",
      "你好<|assistant|>\n",
      "<think></think>\n",
      "你好，我是一个AI助手<|user|>\n",
      "不错～<|assistant|>\n"
     ]
    }
   ],
   "source": [
    "messages = [\n",
    "    {\"role\": \"system\", \"content\": \"You are a helpful assistant.\"},\n",
    "    {\"role\": \"user\", \"content\": \"你好\"},\n",
    "    {\"role\": \"assistant\", \"content\": \"你好，我是一个AI助手\"},\n",
    "    {\"role\": \"user\", \"content\": \"不错～\"},\n",
    "]\n",
    "\n",
    "text = tokenizer.apply_chat_template(\n",
    "    messages,\n",
    "    tokenize=False,\n",
    "    add_generation_prompt=True,\n",
    "    enable_thinking=True\n",
    ")\n",
    "print(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "2503a5fa-9621-4495-9035-8e7ef6525691",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "def process_func(example):\n",
    "    MAX_LENGTH = 1024 # 设置最大序列长度为1024个token\n",
    "    input_ids, attention_mask, labels = [], [], [] # 初始化返回值\n",
    "    # 适配chat_template\n",
    "    instruction = tokenizer(\n",
    "        f\"[gMASK]<sop><|system|>\\n现在你要扮演皇帝身边的女人--甄嬛\" \n",
    "        f\"<|user|>\\n{example['instruction'] + example['input']}\"  \n",
    "        f\"<|assistant|>\\n<think></think>\\n\",  \n",
    "        add_special_tokens=False   \n",
    "    )\n",
    "    response = tokenizer(f\"{example['output']}\", add_special_tokens=False)\n",
    "    # 将instructio部分和response部分的input_ids拼接，并在末尾添加eos token作为标记结束的token\n",
    "    input_ids = instruction[\"input_ids\"] + response[\"input_ids\"]\n",
    "    # 注意力掩码，表示模型需要关注的位置\n",
    "    attention_mask = instruction[\"attention_mask\"] + response[\"attention_mask\"]\n",
    "    # 对于instruction，使用-100表示这些位置不计算loss（即模型不需要预测这部分）\n",
    "    labels = [-100] * len(instruction[\"input_ids\"]) + response[\"input_ids\"]\n",
    "    if len(input_ids) > MAX_LENGTH:  # 超出最大序列长度截断\n",
    "        input_ids = input_ids[:MAX_LENGTH]\n",
    "        attention_mask = attention_mask[:MAX_LENGTH]\n",
    "        labels = labels[:MAX_LENGTH]\n",
    "    return {\n",
    "        \"input_ids\": input_ids,\n",
    "        \"attention_mask\": attention_mask,\n",
    "        \"labels\": labels\n",
    "    }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "84f870d6-73a9-4b0f-8abf-687b32224ad8",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Map: 100%|██████████| 3729/3729 [00:01<00:00, 2785.39 examples/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "Dataset({\n",
       "    features: ['input_ids', 'attention_mask', 'labels'],\n",
       "    num_rows: 3729\n",
       "})"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenized_id = ds.map(process_func, remove_columns=ds.column_names)\n",
    "tokenized_id"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "1f7e15a0-4d9a-4935-9861-00cc472654b1",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[gMASK]<sop><|system|>\n",
      "现在你要扮演皇帝身边的女人--甄嬛<|user|>\n",
      "小姐，别的秀女都在求中选，唯有咱们小姐想被撂牌子，菩萨一定记得真真儿的——<|assistant|>\n",
      "<think></think>\n",
      "嘘——都说许愿说破是不灵的。\n"
     ]
    }
   ],
   "source": [
    "print(tokenizer.decode(tokenized_id[0]['input_ids']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "97f16f66-324a-454f-8cc3-ef23b100ecff",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "你们俩话太多了，我该和温太医要一剂药，好好治治你们。\n"
     ]
    }
   ],
   "source": [
    "print(tokenizer.decode(list(filter(lambda x: x != -100, tokenized_id[1][\"labels\"]))))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "424823a8-ed0d-4309-83c8-3f6b1cdf274c",
   "metadata": {},
   "source": [
    "# 创建模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "170764e5-d899-4ef4-8c53-36f6dec0d198",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loading checkpoint shards: 100%|██████████| 47/47 [00:45<00:00,  1.04it/s]\n",
      "Some weights of the model checkpoint at /model/ModelScope/ZhipuAI/GLM-4.5-Air were not used when initializing Glm4MoeForCausalLM: ['model.layers.46.eh_proj.weight', 'model.layers.46.embed_tokens.weight', 'model.layers.46.enorm.weight', 'model.layers.46.hnorm.weight', 'model.layers.46.input_layernorm.weight', 'model.layers.46.mlp.experts.0.down_proj.weight', 'model.layers.46.mlp.experts.0.gate_proj.weight', 'model.layers.46.mlp.experts.0.up_proj.weight', 'model.layers.46.mlp.experts.1.down_proj.weight', 'model.layers.46.mlp.experts.1.gate_proj.weight', 'model.layers.46.mlp.experts.1.up_proj.weight', 'model.layers.46.mlp.experts.10.down_proj.weight', 'model.layers.46.mlp.experts.10.gate_proj.weight', 'model.layers.46.mlp.experts.10.up_proj.weight', 'model.layers.46.mlp.experts.100.down_proj.weight', 'model.layers.46.mlp.experts.100.gate_proj.weight', 'model.layers.46.mlp.experts.100.up_proj.weight', 'model.layers.46.mlp.experts.101.down_proj.weight', 'model.layers.46.mlp.experts.101.gate_proj.weight', 'model.layers.46.mlp.experts.101.up_proj.weight', 'model.layers.46.mlp.experts.102.down_proj.weight', 'model.layers.46.mlp.experts.102.gate_proj.weight', 'model.layers.46.mlp.experts.102.up_proj.weight', 'model.layers.46.mlp.experts.103.down_proj.weight', 'model.layers.46.mlp.experts.103.gate_proj.weight', 'model.layers.46.mlp.experts.103.up_proj.weight', 'model.layers.46.mlp.experts.104.down_proj.weight', 'model.layers.46.mlp.experts.104.gate_proj.weight', 'model.layers.46.mlp.experts.104.up_proj.weight', 'model.layers.46.mlp.experts.105.down_proj.weight', 'model.layers.46.mlp.experts.105.gate_proj.weight', 'model.layers.46.mlp.experts.105.up_proj.weight', 'model.layers.46.mlp.experts.106.down_proj.weight', 'model.layers.46.mlp.experts.106.gate_proj.weight', 'model.layers.46.mlp.experts.106.up_proj.weight', 'model.layers.46.mlp.experts.107.down_proj.weight', 'model.layers.46.mlp.experts.107.gate_proj.weight', 'model.layers.46.mlp.experts.107.up_proj.weight', 'model.layers.46.mlp.experts.108.down_proj.weight', 'model.layers.46.mlp.experts.108.gate_proj.weight', 'model.layers.46.mlp.experts.108.up_proj.weight', 'model.layers.46.mlp.experts.109.down_proj.weight', 'model.layers.46.mlp.experts.109.gate_proj.weight', 'model.layers.46.mlp.experts.109.up_proj.weight', 'model.layers.46.mlp.experts.11.down_proj.weight', 'model.layers.46.mlp.experts.11.gate_proj.weight', 'model.layers.46.mlp.experts.11.up_proj.weight', 'model.layers.46.mlp.experts.110.down_proj.weight', 'model.layers.46.mlp.experts.110.gate_proj.weight', 'model.layers.46.mlp.experts.110.up_proj.weight', 'model.layers.46.mlp.experts.111.down_proj.weight', 'model.layers.46.mlp.experts.111.gate_proj.weight', 'model.layers.46.mlp.experts.111.up_proj.weight', 'model.layers.46.mlp.experts.112.down_proj.weight', 'model.layers.46.mlp.experts.112.gate_proj.weight', 'model.layers.46.mlp.experts.112.up_proj.weight', 'model.layers.46.mlp.experts.113.down_proj.weight', 'model.layers.46.mlp.experts.113.gate_proj.weight', 'model.layers.46.mlp.experts.113.up_proj.weight', 'model.layers.46.mlp.experts.114.down_proj.weight', 'model.layers.46.mlp.experts.114.gate_proj.weight', 'model.layers.46.mlp.experts.114.up_proj.weight', 'model.layers.46.mlp.experts.115.down_proj.weight', 'model.layers.46.mlp.experts.115.gate_proj.weight', 'model.layers.46.mlp.experts.115.up_proj.weight', 'model.layers.46.mlp.experts.116.down_proj.weight', 'model.layers.46.mlp.experts.116.gate_proj.weight', 'model.layers.46.mlp.experts.116.up_proj.weight', 'model.layers.46.mlp.experts.117.down_proj.weight', 'model.layers.46.mlp.experts.117.gate_proj.weight', 'model.layers.46.mlp.experts.117.up_proj.weight', 'model.layers.46.mlp.experts.118.down_proj.weight', 'model.layers.46.mlp.experts.118.gate_proj.weight', 'model.layers.46.mlp.experts.118.up_proj.weight', 'model.layers.46.mlp.experts.119.down_proj.weight', 'model.layers.46.mlp.experts.119.gate_proj.weight', 'model.layers.46.mlp.experts.119.up_proj.weight', 'model.layers.46.mlp.experts.12.down_proj.weight', 'model.layers.46.mlp.experts.12.gate_proj.weight', 'model.layers.46.mlp.experts.12.up_proj.weight', 'model.layers.46.mlp.experts.120.down_proj.weight', 'model.layers.46.mlp.experts.120.gate_proj.weight', 'model.layers.46.mlp.experts.120.up_proj.weight', 'model.layers.46.mlp.experts.121.down_proj.weight', 'model.layers.46.mlp.experts.121.gate_proj.weight', 'model.layers.46.mlp.experts.121.up_proj.weight', 'model.layers.46.mlp.experts.122.down_proj.weight', 'model.layers.46.mlp.experts.122.gate_proj.weight', 'model.layers.46.mlp.experts.122.up_proj.weight', 'model.layers.46.mlp.experts.123.down_proj.weight', 'model.layers.46.mlp.experts.123.gate_proj.weight', 'model.layers.46.mlp.experts.123.up_proj.weight', 'model.layers.46.mlp.experts.124.down_proj.weight', 'model.layers.46.mlp.experts.124.gate_proj.weight', 'model.layers.46.mlp.experts.124.up_proj.weight', 'model.layers.46.mlp.experts.125.down_proj.weight', 'model.layers.46.mlp.experts.125.gate_proj.weight', 'model.layers.46.mlp.experts.125.up_proj.weight', 'model.layers.46.mlp.experts.126.down_proj.weight', 'model.layers.46.mlp.experts.126.gate_proj.weight', 'model.layers.46.mlp.experts.126.up_proj.weight', 'model.layers.46.mlp.experts.127.down_proj.weight', 'model.layers.46.mlp.experts.127.gate_proj.weight', 'model.layers.46.mlp.experts.127.up_proj.weight', 'model.layers.46.mlp.experts.13.down_proj.weight', 'model.layers.46.mlp.experts.13.gate_proj.weight', 'model.layers.46.mlp.experts.13.up_proj.weight', 'model.layers.46.mlp.experts.14.down_proj.weight', 'model.layers.46.mlp.experts.14.gate_proj.weight', 'model.layers.46.mlp.experts.14.up_proj.weight', 'model.layers.46.mlp.experts.15.down_proj.weight', 'model.layers.46.mlp.experts.15.gate_proj.weight', 'model.layers.46.mlp.experts.15.up_proj.weight', 'model.layers.46.mlp.experts.16.down_proj.weight', 'model.layers.46.mlp.experts.16.gate_proj.weight', 'model.layers.46.mlp.experts.16.up_proj.weight', 'model.layers.46.mlp.experts.17.down_proj.weight', 'model.layers.46.mlp.experts.17.gate_proj.weight', 'model.layers.46.mlp.experts.17.up_proj.weight', 'model.layers.46.mlp.experts.18.down_proj.weight', 'model.layers.46.mlp.experts.18.gate_proj.weight', 'model.layers.46.mlp.experts.18.up_proj.weight', 'model.layers.46.mlp.experts.19.down_proj.weight', 'model.layers.46.mlp.experts.19.gate_proj.weight', 'model.layers.46.mlp.experts.19.up_proj.weight', 'model.layers.46.mlp.experts.2.down_proj.weight', 'model.layers.46.mlp.experts.2.gate_proj.weight', 'model.layers.46.mlp.experts.2.up_proj.weight', 'model.layers.46.mlp.experts.20.down_proj.weight', 'model.layers.46.mlp.experts.20.gate_proj.weight', 'model.layers.46.mlp.experts.20.up_proj.weight', 'model.layers.46.mlp.experts.21.down_proj.weight', 'model.layers.46.mlp.experts.21.gate_proj.weight', 'model.layers.46.mlp.experts.21.up_proj.weight', 'model.layers.46.mlp.experts.22.down_proj.weight', 'model.layers.46.mlp.experts.22.gate_proj.weight', 'model.layers.46.mlp.experts.22.up_proj.weight', 'model.layers.46.mlp.experts.23.down_proj.weight', 'model.layers.46.mlp.experts.23.gate_proj.weight', 'model.layers.46.mlp.experts.23.up_proj.weight', 'model.layers.46.mlp.experts.24.down_proj.weight', 'model.layers.46.mlp.experts.24.gate_proj.weight', 'model.layers.46.mlp.experts.24.up_proj.weight', 'model.layers.46.mlp.experts.25.down_proj.weight', 'model.layers.46.mlp.experts.25.gate_proj.weight', 'model.layers.46.mlp.experts.25.up_proj.weight', 'model.layers.46.mlp.experts.26.down_proj.weight', 'model.layers.46.mlp.experts.26.gate_proj.weight', 'model.layers.46.mlp.experts.26.up_proj.weight', 'model.layers.46.mlp.experts.27.down_proj.weight', 'model.layers.46.mlp.experts.27.gate_proj.weight', 'model.layers.46.mlp.experts.27.up_proj.weight', 'model.layers.46.mlp.experts.28.down_proj.weight', 'model.layers.46.mlp.experts.28.gate_proj.weight', 'model.layers.46.mlp.experts.28.up_proj.weight', 'model.layers.46.mlp.experts.29.down_proj.weight', 'model.layers.46.mlp.experts.29.gate_proj.weight', 'model.layers.46.mlp.experts.29.up_proj.weight', 'model.layers.46.mlp.experts.3.down_proj.weight', 'model.layers.46.mlp.experts.3.gate_proj.weight', 'model.layers.46.mlp.experts.3.up_proj.weight', 'model.layers.46.mlp.experts.30.down_proj.weight', 'model.layers.46.mlp.experts.30.gate_proj.weight', 'model.layers.46.mlp.experts.30.up_proj.weight', 'model.layers.46.mlp.experts.31.down_proj.weight', 'model.layers.46.mlp.experts.31.gate_proj.weight', 'model.layers.46.mlp.experts.31.up_proj.weight', 'model.layers.46.mlp.experts.32.down_proj.weight', 'model.layers.46.mlp.experts.32.gate_proj.weight', 'model.layers.46.mlp.experts.32.up_proj.weight', 'model.layers.46.mlp.experts.33.down_proj.weight', 'model.layers.46.mlp.experts.33.gate_proj.weight', 'model.layers.46.mlp.experts.33.up_proj.weight', 'model.layers.46.mlp.experts.34.down_proj.weight', 'model.layers.46.mlp.experts.34.gate_proj.weight', 'model.layers.46.mlp.experts.34.up_proj.weight', 'model.layers.46.mlp.experts.35.down_proj.weight', 'model.layers.46.mlp.experts.35.gate_proj.weight', 'model.layers.46.mlp.experts.35.up_proj.weight', 'model.layers.46.mlp.experts.36.down_proj.weight', 'model.layers.46.mlp.experts.36.gate_proj.weight', 'model.layers.46.mlp.experts.36.up_proj.weight', 'model.layers.46.mlp.experts.37.down_proj.weight', 'model.layers.46.mlp.experts.37.gate_proj.weight', 'model.layers.46.mlp.experts.37.up_proj.weight', 'model.layers.46.mlp.experts.38.down_proj.weight', 'model.layers.46.mlp.experts.38.gate_proj.weight', 'model.layers.46.mlp.experts.38.up_proj.weight', 'model.layers.46.mlp.experts.39.down_proj.weight', 'model.layers.46.mlp.experts.39.gate_proj.weight', 'model.layers.46.mlp.experts.39.up_proj.weight', 'model.layers.46.mlp.experts.4.down_proj.weight', 'model.layers.46.mlp.experts.4.gate_proj.weight', 'model.layers.46.mlp.experts.4.up_proj.weight', 'model.layers.46.mlp.experts.40.down_proj.weight', 'model.layers.46.mlp.experts.40.gate_proj.weight', 'model.layers.46.mlp.experts.40.up_proj.weight', 'model.layers.46.mlp.experts.41.down_proj.weight', 'model.layers.46.mlp.experts.41.gate_proj.weight', 'model.layers.46.mlp.experts.41.up_proj.weight', 'model.layers.46.mlp.experts.42.down_proj.weight', 'model.layers.46.mlp.experts.42.gate_proj.weight', 'model.layers.46.mlp.experts.42.up_proj.weight', 'model.layers.46.mlp.experts.43.down_proj.weight', 'model.layers.46.mlp.experts.43.gate_proj.weight', 'model.layers.46.mlp.experts.43.up_proj.weight', 'model.layers.46.mlp.experts.44.down_proj.weight', 'model.layers.46.mlp.experts.44.gate_proj.weight', 'model.layers.46.mlp.experts.44.up_proj.weight', 'model.layers.46.mlp.experts.45.down_proj.weight', 'model.layers.46.mlp.experts.45.gate_proj.weight', 'model.layers.46.mlp.experts.45.up_proj.weight', 'model.layers.46.mlp.experts.46.down_proj.weight', 'model.layers.46.mlp.experts.46.gate_proj.weight', 'model.layers.46.mlp.experts.46.up_proj.weight', 'model.layers.46.mlp.experts.47.down_proj.weight', 'model.layers.46.mlp.experts.47.gate_proj.weight', 'model.layers.46.mlp.experts.47.up_proj.weight', 'model.layers.46.mlp.experts.48.down_proj.weight', 'model.layers.46.mlp.experts.48.gate_proj.weight', 'model.layers.46.mlp.experts.48.up_proj.weight', 'model.layers.46.mlp.experts.49.down_proj.weight', 'model.layers.46.mlp.experts.49.gate_proj.weight', 'model.layers.46.mlp.experts.49.up_proj.weight', 'model.layers.46.mlp.experts.5.down_proj.weight', 'model.layers.46.mlp.experts.5.gate_proj.weight', 'model.layers.46.mlp.experts.5.up_proj.weight', 'model.layers.46.mlp.experts.50.down_proj.weight', 'model.layers.46.mlp.experts.50.gate_proj.weight', 'model.layers.46.mlp.experts.50.up_proj.weight', 'model.layers.46.mlp.experts.51.down_proj.weight', 'model.layers.46.mlp.experts.51.gate_proj.weight', 'model.layers.46.mlp.experts.51.up_proj.weight', 'model.layers.46.mlp.experts.52.down_proj.weight', 'model.layers.46.mlp.experts.52.gate_proj.weight', 'model.layers.46.mlp.experts.52.up_proj.weight', 'model.layers.46.mlp.experts.53.down_proj.weight', 'model.layers.46.mlp.experts.53.gate_proj.weight', 'model.layers.46.mlp.experts.53.up_proj.weight', 'model.layers.46.mlp.experts.54.down_proj.weight', 'model.layers.46.mlp.experts.54.gate_proj.weight', 'model.layers.46.mlp.experts.54.up_proj.weight', 'model.layers.46.mlp.experts.55.down_proj.weight', 'model.layers.46.mlp.experts.55.gate_proj.weight', 'model.layers.46.mlp.experts.55.up_proj.weight', 'model.layers.46.mlp.experts.56.down_proj.weight', 'model.layers.46.mlp.experts.56.gate_proj.weight', 'model.layers.46.mlp.experts.56.up_proj.weight', 'model.layers.46.mlp.experts.57.down_proj.weight', 'model.layers.46.mlp.experts.57.gate_proj.weight', 'model.layers.46.mlp.experts.57.up_proj.weight', 'model.layers.46.mlp.experts.58.down_proj.weight', 'model.layers.46.mlp.experts.58.gate_proj.weight', 'model.layers.46.mlp.experts.58.up_proj.weight', 'model.layers.46.mlp.experts.59.down_proj.weight', 'model.layers.46.mlp.experts.59.gate_proj.weight', 'model.layers.46.mlp.experts.59.up_proj.weight', 'model.layers.46.mlp.experts.6.down_proj.weight', 'model.layers.46.mlp.experts.6.gate_proj.weight', 'model.layers.46.mlp.experts.6.up_proj.weight', 'model.layers.46.mlp.experts.60.down_proj.weight', 'model.layers.46.mlp.experts.60.gate_proj.weight', 'model.layers.46.mlp.experts.60.up_proj.weight', 'model.layers.46.mlp.experts.61.down_proj.weight', 'model.layers.46.mlp.experts.61.gate_proj.weight', 'model.layers.46.mlp.experts.61.up_proj.weight', 'model.layers.46.mlp.experts.62.down_proj.weight', 'model.layers.46.mlp.experts.62.gate_proj.weight', 'model.layers.46.mlp.experts.62.up_proj.weight', 'model.layers.46.mlp.experts.63.down_proj.weight', 'model.layers.46.mlp.experts.63.gate_proj.weight', 'model.layers.46.mlp.experts.63.up_proj.weight', 'model.layers.46.mlp.experts.64.down_proj.weight', 'model.layers.46.mlp.experts.64.gate_proj.weight', 'model.layers.46.mlp.experts.64.up_proj.weight', 'model.layers.46.mlp.experts.65.down_proj.weight', 'model.layers.46.mlp.experts.65.gate_proj.weight', 'model.layers.46.mlp.experts.65.up_proj.weight', 'model.layers.46.mlp.experts.66.down_proj.weight', 'model.layers.46.mlp.experts.66.gate_proj.weight', 'model.layers.46.mlp.experts.66.up_proj.weight', 'model.layers.46.mlp.experts.67.down_proj.weight', 'model.layers.46.mlp.experts.67.gate_proj.weight', 'model.layers.46.mlp.experts.67.up_proj.weight', 'model.layers.46.mlp.experts.68.down_proj.weight', 'model.layers.46.mlp.experts.68.gate_proj.weight', 'model.layers.46.mlp.experts.68.up_proj.weight', 'model.layers.46.mlp.experts.69.down_proj.weight', 'model.layers.46.mlp.experts.69.gate_proj.weight', 'model.layers.46.mlp.experts.69.up_proj.weight', 'model.layers.46.mlp.experts.7.down_proj.weight', 'model.layers.46.mlp.experts.7.gate_proj.weight', 'model.layers.46.mlp.experts.7.up_proj.weight', 'model.layers.46.mlp.experts.70.down_proj.weight', 'model.layers.46.mlp.experts.70.gate_proj.weight', 'model.layers.46.mlp.experts.70.up_proj.weight', 'model.layers.46.mlp.experts.71.down_proj.weight', 'model.layers.46.mlp.experts.71.gate_proj.weight', 'model.layers.46.mlp.experts.71.up_proj.weight', 'model.layers.46.mlp.experts.72.down_proj.weight', 'model.layers.46.mlp.experts.72.gate_proj.weight', 'model.layers.46.mlp.experts.72.up_proj.weight', 'model.layers.46.mlp.experts.73.down_proj.weight', 'model.layers.46.mlp.experts.73.gate_proj.weight', 'model.layers.46.mlp.experts.73.up_proj.weight', 'model.layers.46.mlp.experts.74.down_proj.weight', 'model.layers.46.mlp.experts.74.gate_proj.weight', 'model.layers.46.mlp.experts.74.up_proj.weight', 'model.layers.46.mlp.experts.75.down_proj.weight', 'model.layers.46.mlp.experts.75.gate_proj.weight', 'model.layers.46.mlp.experts.75.up_proj.weight', 'model.layers.46.mlp.experts.76.down_proj.weight', 'model.layers.46.mlp.experts.76.gate_proj.weight', 'model.layers.46.mlp.experts.76.up_proj.weight', 'model.layers.46.mlp.experts.77.down_proj.weight', 'model.layers.46.mlp.experts.77.gate_proj.weight', 'model.layers.46.mlp.experts.77.up_proj.weight', 'model.layers.46.mlp.experts.78.down_proj.weight', 'model.layers.46.mlp.experts.78.gate_proj.weight', 'model.layers.46.mlp.experts.78.up_proj.weight', 'model.layers.46.mlp.experts.79.down_proj.weight', 'model.layers.46.mlp.experts.79.gate_proj.weight', 'model.layers.46.mlp.experts.79.up_proj.weight', 'model.layers.46.mlp.experts.8.down_proj.weight', 'model.layers.46.mlp.experts.8.gate_proj.weight', 'model.layers.46.mlp.experts.8.up_proj.weight', 'model.layers.46.mlp.experts.80.down_proj.weight', 'model.layers.46.mlp.experts.80.gate_proj.weight', 'model.layers.46.mlp.experts.80.up_proj.weight', 'model.layers.46.mlp.experts.81.down_proj.weight', 'model.layers.46.mlp.experts.81.gate_proj.weight', 'model.layers.46.mlp.experts.81.up_proj.weight', 'model.layers.46.mlp.experts.82.down_proj.weight', 'model.layers.46.mlp.experts.82.gate_proj.weight', 'model.layers.46.mlp.experts.82.up_proj.weight', 'model.layers.46.mlp.experts.83.down_proj.weight', 'model.layers.46.mlp.experts.83.gate_proj.weight', 'model.layers.46.mlp.experts.83.up_proj.weight', 'model.layers.46.mlp.experts.84.down_proj.weight', 'model.layers.46.mlp.experts.84.gate_proj.weight', 'model.layers.46.mlp.experts.84.up_proj.weight', 'model.layers.46.mlp.experts.85.down_proj.weight', 'model.layers.46.mlp.experts.85.gate_proj.weight', 'model.layers.46.mlp.experts.85.up_proj.weight', 'model.layers.46.mlp.experts.86.down_proj.weight', 'model.layers.46.mlp.experts.86.gate_proj.weight', 'model.layers.46.mlp.experts.86.up_proj.weight', 'model.layers.46.mlp.experts.87.down_proj.weight', 'model.layers.46.mlp.experts.87.gate_proj.weight', 'model.layers.46.mlp.experts.87.up_proj.weight', 'model.layers.46.mlp.experts.88.down_proj.weight', 'model.layers.46.mlp.experts.88.gate_proj.weight', 'model.layers.46.mlp.experts.88.up_proj.weight', 'model.layers.46.mlp.experts.89.down_proj.weight', 'model.layers.46.mlp.experts.89.gate_proj.weight', 'model.layers.46.mlp.experts.89.up_proj.weight', 'model.layers.46.mlp.experts.9.down_proj.weight', 'model.layers.46.mlp.experts.9.gate_proj.weight', 'model.layers.46.mlp.experts.9.up_proj.weight', 'model.layers.46.mlp.experts.90.down_proj.weight', 'model.layers.46.mlp.experts.90.gate_proj.weight', 'model.layers.46.mlp.experts.90.up_proj.weight', 'model.layers.46.mlp.experts.91.down_proj.weight', 'model.layers.46.mlp.experts.91.gate_proj.weight', 'model.layers.46.mlp.experts.91.up_proj.weight', 'model.layers.46.mlp.experts.92.down_proj.weight', 'model.layers.46.mlp.experts.92.gate_proj.weight', 'model.layers.46.mlp.experts.92.up_proj.weight', 'model.layers.46.mlp.experts.93.down_proj.weight', 'model.layers.46.mlp.experts.93.gate_proj.weight', 'model.layers.46.mlp.experts.93.up_proj.weight', 'model.layers.46.mlp.experts.94.down_proj.weight', 'model.layers.46.mlp.experts.94.gate_proj.weight', 'model.layers.46.mlp.experts.94.up_proj.weight', 'model.layers.46.mlp.experts.95.down_proj.weight', 'model.layers.46.mlp.experts.95.gate_proj.weight', 'model.layers.46.mlp.experts.95.up_proj.weight', 'model.layers.46.mlp.experts.96.down_proj.weight', 'model.layers.46.mlp.experts.96.gate_proj.weight', 'model.layers.46.mlp.experts.96.up_proj.weight', 'model.layers.46.mlp.experts.97.down_proj.weight', 'model.layers.46.mlp.experts.97.gate_proj.weight', 'model.layers.46.mlp.experts.97.up_proj.weight', 'model.layers.46.mlp.experts.98.down_proj.weight', 'model.layers.46.mlp.experts.98.gate_proj.weight', 'model.layers.46.mlp.experts.98.up_proj.weight', 'model.layers.46.mlp.experts.99.down_proj.weight', 'model.layers.46.mlp.experts.99.gate_proj.weight', 'model.layers.46.mlp.experts.99.up_proj.weight', 'model.layers.46.mlp.gate.e_score_correction_bias', 'model.layers.46.mlp.gate.weight', 'model.layers.46.mlp.shared_experts.down_proj.weight', 'model.layers.46.mlp.shared_experts.gate_proj.weight', 'model.layers.46.mlp.shared_experts.up_proj.weight', 'model.layers.46.post_attention_layernorm.weight', 'model.layers.46.self_attn.k_proj.bias', 'model.layers.46.self_attn.k_proj.weight', 'model.layers.46.self_attn.o_proj.weight', 'model.layers.46.self_attn.q_proj.bias', 'model.layers.46.self_attn.q_proj.weight', 'model.layers.46.self_attn.v_proj.bias', 'model.layers.46.self_attn.v_proj.weight', 'model.layers.46.shared_head.head.weight', 'model.layers.46.shared_head.norm.weight']\n",
      "- This IS expected if you are initializing Glm4MoeForCausalLM from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing Glm4MoeForCausalLM from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "Glm4MoeForCausalLM(\n",
       "  (model): Glm4MoeModel(\n",
       "    (embed_tokens): Embedding(151552, 4096, padding_idx=151329)\n",
       "    (layers): ModuleList(\n",
       "      (0): Glm4MoeDecoderLayer(\n",
       "        (self_attn): Glm4MoeAttention(\n",
       "          (q_proj): Linear(in_features=4096, out_features=12288, bias=True)\n",
       "          (k_proj): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "          (v_proj): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "          (o_proj): Linear(in_features=12288, out_features=4096, bias=False)\n",
       "        )\n",
       "        (mlp): Glm4MoeMLP(\n",
       "          (gate_proj): Linear(in_features=4096, out_features=10944, bias=False)\n",
       "          (up_proj): Linear(in_features=4096, out_features=10944, bias=False)\n",
       "          (down_proj): Linear(in_features=10944, out_features=4096, bias=False)\n",
       "          (act_fn): SiLU()\n",
       "        )\n",
       "        (input_layernorm): Glm4MoeRMSNorm((4096,), eps=1e-05)\n",
       "        (post_attention_layernorm): Glm4MoeRMSNorm((4096,), eps=1e-05)\n",
       "      )\n",
       "      (1-45): 45 x Glm4MoeDecoderLayer(\n",
       "        (self_attn): Glm4MoeAttention(\n",
       "          (q_proj): Linear(in_features=4096, out_features=12288, bias=True)\n",
       "          (k_proj): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "          (v_proj): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "          (o_proj): Linear(in_features=12288, out_features=4096, bias=False)\n",
       "        )\n",
       "        (mlp): Glm4MoeMoE(\n",
       "          (experts): ModuleList(\n",
       "            (0-127): 128 x Glm4MoeMLP(\n",
       "              (gate_proj): Linear(in_features=4096, out_features=1408, bias=False)\n",
       "              (up_proj): Linear(in_features=4096, out_features=1408, bias=False)\n",
       "              (down_proj): Linear(in_features=1408, out_features=4096, bias=False)\n",
       "              (act_fn): SiLU()\n",
       "            )\n",
       "          )\n",
       "          (gate): Glm4MoeTopkRouter()\n",
       "          (shared_experts): Glm4MoeMLP(\n",
       "            (gate_proj): Linear(in_features=4096, out_features=1408, bias=False)\n",
       "            (up_proj): Linear(in_features=4096, out_features=1408, bias=False)\n",
       "            (down_proj): Linear(in_features=1408, out_features=4096, bias=False)\n",
       "            (act_fn): SiLU()\n",
       "          )\n",
       "        )\n",
       "        (input_layernorm): Glm4MoeRMSNorm((4096,), eps=1e-05)\n",
       "        (post_attention_layernorm): Glm4MoeRMSNorm((4096,), eps=1e-05)\n",
       "      )\n",
       "    )\n",
       "    (norm): Glm4MoeRMSNorm((4096,), eps=1e-05)\n",
       "    (rotary_emb): Glm4MoeRotaryEmbedding()\n",
       "  )\n",
       "  (lm_head): Linear(in_features=4096, out_features=151552, bias=False)\n",
       ")"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import torch\n",
    "\n",
    "model = AutoModelForCausalLM.from_pretrained('/model/ModelScope/ZhipuAI/GLM-4.5-Air', device_map=\"auto\",torch_dtype=torch.bfloat16)\n",
    "model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "2323eac7-37d5-4288-8bc5-79fac7113402",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "model.enable_input_require_grads() # 开启梯度检查点时，要执行该方法"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "f808b05c-f2cb-48cf-a80d-0c42be6051c7",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.bfloat16"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.dtype"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "13d71257-3c1c-4303-8ff8-af161ebc2cf1",
   "metadata": {},
   "source": [
    "# lora "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "2d304ae2-ab60-4080-a80d-19cac2e3ade3",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LoraConfig(task_type=<TaskType.CAUSAL_LM: 'CAUSAL_LM'>, peft_type=<PeftType.LORA: 'LORA'>, auto_mapping=None, base_model_name_or_path=None, revision=None, inference_mode=False, r=8, target_modules={'up_proj', 'q_proj', 'down_proj', 'o_proj', 'gate_proj', 'k_proj', 'v_proj'}, exclude_modules=None, lora_alpha=32, lora_dropout=0.1, fan_in_fan_out=False, bias='none', use_rslora=False, modules_to_save=None, init_lora_weights=True, layers_to_transform=None, layers_pattern=None, rank_pattern={}, alpha_pattern={}, megatron_config=None, megatron_core='megatron.core', trainable_token_indices=None, loftq_config={}, eva_config=None, corda_config=None, use_dora=False, use_qalora=False, qalora_group_size=16, layer_replication=None, runtime_config=LoraRuntimeConfig(ephemeral_gpu_offload=False), lora_bias=False)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from peft import LoraConfig, TaskType, get_peft_model\n",
    "\n",
    "config = LoraConfig(\n",
    "    task_type=TaskType.CAUSAL_LM, \n",
    "    target_modules=[\"q_proj\", \"k_proj\", \"v_proj\", \"o_proj\", \"gate_proj\", \"up_proj\", \"down_proj\"],\n",
    "    inference_mode=False, # 训练模式\n",
    "    r=8, # Lora 秩\n",
    "    lora_alpha=32, # Lora alaph，具体作用参见 Lora 原理\n",
    "    lora_dropout=0.1# Dropout 比例\n",
    ")\n",
    "config"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "2c2489c5-eaab-4e1f-b06a-c3f914b4bf8e",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LoraConfig(task_type=<TaskType.CAUSAL_LM: 'CAUSAL_LM'>, peft_type=<PeftType.LORA: 'LORA'>, auto_mapping=None, base_model_name_or_path='/model/ModelScope/ZhipuAI/GLM-4.5-Air', revision=None, inference_mode=False, r=8, target_modules={'up_proj', 'q_proj', 'down_proj', 'o_proj', 'gate_proj', 'k_proj', 'v_proj'}, exclude_modules=None, lora_alpha=32, lora_dropout=0.1, fan_in_fan_out=False, bias='none', use_rslora=False, modules_to_save=None, init_lora_weights=True, layers_to_transform=None, layers_pattern=None, rank_pattern={}, alpha_pattern={}, megatron_config=None, megatron_core='megatron.core', trainable_token_indices=None, loftq_config={}, eva_config=None, corda_config=None, use_dora=False, use_qalora=False, qalora_group_size=16, layer_replication=None, runtime_config=LoraRuntimeConfig(ephemeral_gpu_offload=False), lora_bias=False)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = get_peft_model(model, config)\n",
    "config"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "ebf5482b-fab9-4eb3-ad88-c116def4be12",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "trainable params: 783,005,184 || all params: 107,635,250,688 || trainable%: 0.7275\n"
     ]
    }
   ],
   "source": [
    "model.print_trainable_parameters()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ca055683-837f-4865-9c57-9164ba60c00f",
   "metadata": {},
   "source": [
    "# 配置训练参数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "7e76bbff-15fd-4995-a61d-8364dc5e9ea0",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "args = TrainingArguments(\n",
    "    output_dir=\"./output/glm45_air_lora\",\n",
    "    per_device_train_batch_size=4,\n",
    "    gradient_accumulation_steps=4,\n",
    "    logging_steps=2,\n",
    "    num_train_epochs=3,\n",
    "    save_steps=10, \n",
    "    learning_rate=1e-4,\n",
    "    save_on_each_node=True,\n",
    "    gradient_checkpointing=True,\n",
    "    report_to=\"none\",\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "9faedcb1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">/usr/local/miniconda3/envs/py310/lib/python3.10/site-packages/rich/live.py:231: UserWarning: install \"ipywidgets\" \n",
       "for Jupyter support\n",
       "  warnings.warn('install \"ipywidgets\" for Jupyter support')\n",
       "</pre>\n"
      ],
      "text/plain": [
       "/usr/local/miniconda3/envs/py310/lib/python3.10/site-packages/rich/live.py:231: UserWarning: install \"ipywidgets\" \n",
       "for Jupyter support\n",
       "  warnings.warn('install \"ipywidgets\" for Jupyter support')\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"></pre>\n"
      ],
      "text/plain": []
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import swanlab\n",
    "from swanlab.integration.transformers import SwanLabCallback\n",
    "\n",
    "swanlab.login(api_key='BIYVGq2rfWmD9sFMCehUG', save=True)\n",
    "\n",
    "# 实例化SwanLabCallback\n",
    "swanlab_callback = SwanLabCallback(\n",
    "    project=\"self-llm\", \n",
    "    experiment_name=\"glm45_air_lora_experiment\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "f142cb9c-ad99-48e6-ba86-6df198f9ed96",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "No label_names provided for model class `PeftModelForCausalLM`. Since `PeftModel` hides base models input arguments, if label_names is not given, label_names can't be set automatically within `Trainer`. Note that empty label_names list will be used instead.\n"
     ]
    }
   ],
   "source": [
    "trainer = Trainer(\n",
    "    model=model,\n",
    "    args=args,\n",
    "    train_dataset=tokenized_id,\n",
    "    data_collator=DataCollatorForSeq2Seq(tokenizer=tokenizer, padding=True),\n",
    "    callbacks=[swanlab_callback]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "aec9bc36-b297-45af-99e1-d4c4d82be081",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"></pre>\n"
      ],
      "text/plain": []
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #000080; text-decoration-color: #000080; font-weight: bold\">swanlab</span><span style=\"color: #000000; text-decoration-color: #000000; font-weight: bold\">:</span> Tracking run with swanlab version <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.6</span>.<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">7</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;34mswanlab\u001b[0m\u001b[1;39m:\u001b[0m Tracking run with swanlab version \u001b[1;36m0.6\u001b[0m.\u001b[1;36m7\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #000080; text-decoration-color: #000080; font-weight: bold\">swanlab</span><span style=\"color: #000000; text-decoration-color: #000000; font-weight: bold\">:</span> Run data will be saved locally in <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">/workspace/swanlog/run-20250729_034415-g2v55jfocwebf6qhaucu3</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;34mswanlab\u001b[0m\u001b[1;39m:\u001b[0m Run data will be saved locally in \u001b[1;35m/workspace/swanlog/run-20250729_034415-g2v55jfocwebf6qhaucu3\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #000080; text-decoration-color: #000080; font-weight: bold\">swanlab</span><span style=\"color: #000000; text-decoration-color: #000000; font-weight: bold\">:</span><span style=\"color: #000080; text-decoration-color: #000080; font-weight: bold\"> </span>👋 Hi <span style=\"color: #000000; text-decoration-color: #000000; font-weight: bold\">kmno4</span>,welcome to swanlab!\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;34mswanlab\u001b[0m\u001b[1;39m:\u001b[0m\u001b[1;34m \u001b[0m👋 Hi \u001b[1;39mkmno4\u001b[0m,welcome to swanlab!\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #000080; text-decoration-color: #000080; font-weight: bold\">swanlab</span><span style=\"color: #000000; text-decoration-color: #000000; font-weight: bold\">:</span> Syncing run <span style=\"color: #808000; text-decoration-color: #808000\">glm45_air_lora_experiment</span> to the cloud\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;34mswanlab\u001b[0m\u001b[1;39m:\u001b[0m Syncing run \u001b[33mglm45_air_lora_experiment\u001b[0m to the cloud\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #000080; text-decoration-color: #000080; font-weight: bold\">swanlab</span><span style=\"color: #000000; text-decoration-color: #000000; font-weight: bold\">:</span> 🏠 View project at <span style=\"color: #000080; text-decoration-color: #000080; text-decoration: underline\">https://swanlab.cn/@kmno4/self-llm</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;34mswanlab\u001b[0m\u001b[1;39m:\u001b[0m 🏠 View project at \u001b[4;34mhttps://swanlab.cn/@kmno4/self-llm\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #000080; text-decoration-color: #000080; font-weight: bold\">swanlab</span><span style=\"color: #000000; text-decoration-color: #000000; font-weight: bold\">:</span> 🚀 View run at <span style=\"color: #000080; text-decoration-color: #000080; text-decoration: underline\">https://swanlab.cn/@kmno4/self-llm/runs/g2v55jfocwebf6qhaucu3</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;34mswanlab\u001b[0m\u001b[1;39m:\u001b[0m 🚀 View run at \u001b[4;34mhttps://swanlab.cn/@kmno4/self-llm/runs/g2v55jfocwebf6qhaucu3\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "<!DOCTYPE html>\n",
       "<html lang=\"en\">\n",
       "<head>\n",
       "    <meta charset=\"UTF-8\">\n",
       "    <meta name=\"viewport\" content=\"width=device-width, initial-scale=1.0\">\n",
       "    <title>Show Iframe</title>\n",
       "    \n",
       "        <script>\n",
       "            function showIframe() {\n",
       "                var iframeHtml = '<iframe src=\"https://swanlab.cn/@kmno4/self-llm/runs/g2v55jfocwebf6qhaucu3\" width=100% height=\"600\" frameborder=\"no\"></iframe>';\n",
       "                document.getElementById('iframeContainer').innerHTML = iframeHtml;\n",
       "            }\n",
       "        </script>\n",
       "        \n",
       "</head>\n",
       "<body>\n",
       "    <style>\n",
       "        .interactive-button {\n",
       "            display: flex;\n",
       "            align-items: center;\n",
       "            height: 36px;\n",
       "            border: 0px;\n",
       "            background-color: #2c8f63;\n",
       "            color: white;\n",
       "            padding: 10px 20px;\n",
       "            transition: background-color 0.3s, transform 0.2s;\n",
       "        }\n",
       "\n",
       "        .interactive-button:hover {\n",
       "            background-color: #5cab87;\n",
       "            cursor: pointer;\n",
       "        }\n",
       "\n",
       "        .interactive-button:active { background-color: #217952; transform: scale(0.96); } </style> <br> <button \n",
       "        onclick=\"showIframe()\" class=\"interactive-button\"> <svg style=\"height: 16px; margin-right: 8px;\" viewBox=\"0 0 \n",
       "        46 46\" fill=\"none\"> <path d=\"M10.8439 21.1974C10.6414 21.2854 10.4477 21.3925 10.2655 21.5173L10.2069 \n",
       "        21.5652C10.1839 21.58 10.1625 21.5969 10.1429 21.6159C6.29135 24.6118 4.22831 29.4416 5.32646 34.282C5.94656 \n",
       "        37.0577 7.50461 39.5348 9.73801 41.2958C11.9714 43.0568 14.7436 43.994 17.5874 43.9495H18.0219C19.8864 \n",
       "        43.8697 21.7087 43.3694 23.3526 42.486C24.9964 41.6026 26.4193 40.3589 27.5147 38.848C28.61 37.3371 29.3496 \n",
       "        35.598 29.678 33.761C30.0065 31.9239 29.9153 30.0363 29.4112 28.2395C28.9181 26.4723 27.8919 24.8437 26.9937 \n",
       "        23.2551C25.4158 20.4653 23.8343 17.6764 22.2492 14.8884C21.7801 14.0647 21.3057 13.2465 20.8419 \n",
       "        12.4228C20.2315 11.3353 19.2746 10.1519 19.224 8.86183C19.1733 7.57176 20.2235 6.32701 21.5082 \n",
       "        6.07912C23.9284 5.61801 25.0639 8.24078 25.0693 8.23812C25.363 8.94035 25.9123 9.50489 26.6063 \n",
       "        9.81764C27.3002 10.1304 28.087 10.168 28.8077 9.92298C29.5283 9.67791 30.1291 9.1684 30.4885 8.49743C30.8479 \n",
       "        7.82646 30.9392 7.04405 30.7439 6.30835C30.1514 4.37314 28.9133 2.69953 27.2363 1.56656C25.7615 0.511704 \n",
       "        23.9847 -0.0372109 22.1719 0.00195984C20.9049 0.00893199 19.6532 0.27989 18.4967 0.797557C17.3402 1.31522 \n",
       "        16.3043 2.06823 15.4551 3.00856C14.49 4.08707 13.7984 5.38193 13.4389 6.78385C13.0794 8.18576 13.0624 9.6536 \n",
       "        13.3894 11.0635C13.52 11.593 13.6984 12.1095 13.9225 12.6067C14.5595 14.0514 15.4951 15.3681 16.284 \n",
       "        16.7355C17.2525 18.4147 18.2209 20.0948 19.1893 21.7758C20.1578 23.4568 21.1351 25.1449 22.1213 \n",
       "        26.8401C22.9209 28.2421 23.7925 29.4682 23.8805 31.1528C23.9175 32.0513 23.7682 32.9479 23.4419 \n",
       "        33.7859C23.1156 34.6239 22.6194 35.3854 21.9845 36.0223C21.3496 36.6592 20.5897 37.1578 19.7527 \n",
       "        37.4868C18.9157 37.8157 18.0196 37.9678 17.121 37.9336C14.0024 37.7923 11.6488 35.4814 11.1744 32.4588C10.58 \n",
       "        28.6419 13.552 26.5469 13.552 26.5469C14.1782 26.1785 14.6497 25.5955 14.8791 24.906C15.1084 24.2166 15.0801 \n",
       "        23.4673 14.7993 22.7971C14.5186 22.127 14.0044 21.5813 13.3521 21.2611C12.6998 20.941 11.9536 20.8682 11.2517 \n",
       "        21.0561C11.1174 21.0939 10.9856 21.1402 10.8572 21.1947\" fill=\"white\" /> <path d=\"M42.8101 31.5968C42.8109 \n",
       "        30.5198 42.7218 29.4445 42.5435 28.3823C42.2663 26.7069 41.7464 25.0808 41.0002 23.5552C40.5524 22.6463 \n",
       "        39.9874 21.7374 39.1024 21.2417C38.6593 20.9919 38.1589 20.8617 37.6502 20.8639C37.1416 20.8661 36.6423 \n",
       "        21.0006 36.2013 21.2541C35.7604 21.5077 35.393 21.8716 35.1352 22.3101C34.8775 22.7485 34.7382 23.2466 \n",
       "        34.7312 23.7552C34.7072 24.8773 35.3149 25.8875 35.768 26.9217C36.5212 28.6453 36.8623 30.5208 36.7642 \n",
       "        32.3993C36.6661 34.2777 36.1315 36.1075 35.2029 37.7433C35.146 37.8404 35.0952 37.941 35.051 38.0445C34.8623 \n",
       "        38.4842 34.7635 38.9573 34.7605 39.4358C34.7802 40.1222 35.0356 40.7808 35.4835 41.3011C35.9315 41.8214 \n",
       "        36.5449 42.1717 37.2207 42.2932C38.8759 42.589 40.1899 41.347 40.8856 39.9609C42.1643 37.3589 42.823 34.4961 \n",
       "        42.8101 31.5968Z\" fill=\"white\" /> <path d=\"M28.2309 11.8938C28.1761 11.9043 28.1218 11.9176 28.0683 \n",
       "        11.9338C27.9593 11.9642 27.8611 12.0249 27.7851 12.1088C27.7091 12.1928 27.6584 12.2965 27.6389 \n",
       "        12.408C27.6193 12.5195 27.6318 12.6343 27.6748 12.7391C27.7178 12.8438 27.7895 12.9343 27.8818 \n",
       "        12.9999C29.2375 14.0252 30.3809 15.3043 31.2482 16.7662C31.4838 17.1677 31.6888 17.5865 31.8612 \n",
       "        18.0189C32.0052 18.3921 32.1971 18.8799 32.6822 18.8532C33.0607 18.8346 33.2153 18.512 33.3192 \n",
       "        18.1895C33.8137 16.5125 33.9678 14.7534 33.7723 13.0159C33.6331 12.0693 33.4155 11.1359 33.122 \n",
       "        10.2252C33.0775 10.0047 32.9744 9.80029 32.8235 9.6335C32.7273 9.54627 32.6054 9.49262 32.4761 9.4806C32.3468 \n",
       "        9.46859 32.2171 9.49886 32.1065 9.56687C32.0016 9.65188 31.9115 9.75365 31.8399 9.86806C31.3956 10.4658 \n",
       "        30.825 10.9581 30.1687 11.3101C29.8377 11.4861 29.4893 11.6272 29.1292 11.7312C28.828 11.8192 28.5215 11.8325 \n",
       "        28.2309 11.8938Z\" fill=\"white\" /> </svg> Display SwanLab Board </button> <br> <div \n",
       "        id=\"iframeContainer\"></div> </body> </html>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='23' max='702' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [ 23/702 23:13 < 12:31:00, 0.02 it/s, Epoch 0.09/3]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>3.537200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>3.383800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>2.945900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>3.113100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10</td>\n",
       "      <td>3.166800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>12</td>\n",
       "      <td>2.749700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>14</td>\n",
       "      <td>3.000500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>16</td>\n",
       "      <td>2.643000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>18</td>\n",
       "      <td>2.921100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>20</td>\n",
       "      <td>3.173600</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "trainer.train()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8abb2327-458e-4e96-ac98-2141b5b97c8e",
   "metadata": {},
   "source": [
    "# 合并加载模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "bd2a415a-a9ad-49ea-877f-243558a83bfc",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/miniconda3/envs/py310/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n",
      "Loading checkpoint shards: 100%|██████████| 47/47 [00:46<00:00,  1.02it/s]\n",
      "Some weights of the model checkpoint at /model/ModelScope/ZhipuAI/GLM-4.5-Air were not used when initializing Glm4MoeForCausalLM: ['model.layers.46.eh_proj.weight', 'model.layers.46.embed_tokens.weight', 'model.layers.46.enorm.weight', 'model.layers.46.hnorm.weight', 'model.layers.46.input_layernorm.weight', 'model.layers.46.mlp.experts.0.down_proj.weight', 'model.layers.46.mlp.experts.0.gate_proj.weight', 'model.layers.46.mlp.experts.0.up_proj.weight', 'model.layers.46.mlp.experts.1.down_proj.weight', 'model.layers.46.mlp.experts.1.gate_proj.weight', 'model.layers.46.mlp.experts.1.up_proj.weight', 'model.layers.46.mlp.experts.10.down_proj.weight', 'model.layers.46.mlp.experts.10.gate_proj.weight', 'model.layers.46.mlp.experts.10.up_proj.weight', 'model.layers.46.mlp.experts.100.down_proj.weight', 'model.layers.46.mlp.experts.100.gate_proj.weight', 'model.layers.46.mlp.experts.100.up_proj.weight', 'model.layers.46.mlp.experts.101.down_proj.weight', 'model.layers.46.mlp.experts.101.gate_proj.weight', 'model.layers.46.mlp.experts.101.up_proj.weight', 'model.layers.46.mlp.experts.102.down_proj.weight', 'model.layers.46.mlp.experts.102.gate_proj.weight', 'model.layers.46.mlp.experts.102.up_proj.weight', 'model.layers.46.mlp.experts.103.down_proj.weight', 'model.layers.46.mlp.experts.103.gate_proj.weight', 'model.layers.46.mlp.experts.103.up_proj.weight', 'model.layers.46.mlp.experts.104.down_proj.weight', 'model.layers.46.mlp.experts.104.gate_proj.weight', 'model.layers.46.mlp.experts.104.up_proj.weight', 'model.layers.46.mlp.experts.105.down_proj.weight', 'model.layers.46.mlp.experts.105.gate_proj.weight', 'model.layers.46.mlp.experts.105.up_proj.weight', 'model.layers.46.mlp.experts.106.down_proj.weight', 'model.layers.46.mlp.experts.106.gate_proj.weight', 'model.layers.46.mlp.experts.106.up_proj.weight', 'model.layers.46.mlp.experts.107.down_proj.weight', 'model.layers.46.mlp.experts.107.gate_proj.weight', 'model.layers.46.mlp.experts.107.up_proj.weight', 'model.layers.46.mlp.experts.108.down_proj.weight', 'model.layers.46.mlp.experts.108.gate_proj.weight', 'model.layers.46.mlp.experts.108.up_proj.weight', 'model.layers.46.mlp.experts.109.down_proj.weight', 'model.layers.46.mlp.experts.109.gate_proj.weight', 'model.layers.46.mlp.experts.109.up_proj.weight', 'model.layers.46.mlp.experts.11.down_proj.weight', 'model.layers.46.mlp.experts.11.gate_proj.weight', 'model.layers.46.mlp.experts.11.up_proj.weight', 'model.layers.46.mlp.experts.110.down_proj.weight', 'model.layers.46.mlp.experts.110.gate_proj.weight', 'model.layers.46.mlp.experts.110.up_proj.weight', 'model.layers.46.mlp.experts.111.down_proj.weight', 'model.layers.46.mlp.experts.111.gate_proj.weight', 'model.layers.46.mlp.experts.111.up_proj.weight', 'model.layers.46.mlp.experts.112.down_proj.weight', 'model.layers.46.mlp.experts.112.gate_proj.weight', 'model.layers.46.mlp.experts.112.up_proj.weight', 'model.layers.46.mlp.experts.113.down_proj.weight', 'model.layers.46.mlp.experts.113.gate_proj.weight', 'model.layers.46.mlp.experts.113.up_proj.weight', 'model.layers.46.mlp.experts.114.down_proj.weight', 'model.layers.46.mlp.experts.114.gate_proj.weight', 'model.layers.46.mlp.experts.114.up_proj.weight', 'model.layers.46.mlp.experts.115.down_proj.weight', 'model.layers.46.mlp.experts.115.gate_proj.weight', 'model.layers.46.mlp.experts.115.up_proj.weight', 'model.layers.46.mlp.experts.116.down_proj.weight', 'model.layers.46.mlp.experts.116.gate_proj.weight', 'model.layers.46.mlp.experts.116.up_proj.weight', 'model.layers.46.mlp.experts.117.down_proj.weight', 'model.layers.46.mlp.experts.117.gate_proj.weight', 'model.layers.46.mlp.experts.117.up_proj.weight', 'model.layers.46.mlp.experts.118.down_proj.weight', 'model.layers.46.mlp.experts.118.gate_proj.weight', 'model.layers.46.mlp.experts.118.up_proj.weight', 'model.layers.46.mlp.experts.119.down_proj.weight', 'model.layers.46.mlp.experts.119.gate_proj.weight', 'model.layers.46.mlp.experts.119.up_proj.weight', 'model.layers.46.mlp.experts.12.down_proj.weight', 'model.layers.46.mlp.experts.12.gate_proj.weight', 'model.layers.46.mlp.experts.12.up_proj.weight', 'model.layers.46.mlp.experts.120.down_proj.weight', 'model.layers.46.mlp.experts.120.gate_proj.weight', 'model.layers.46.mlp.experts.120.up_proj.weight', 'model.layers.46.mlp.experts.121.down_proj.weight', 'model.layers.46.mlp.experts.121.gate_proj.weight', 'model.layers.46.mlp.experts.121.up_proj.weight', 'model.layers.46.mlp.experts.122.down_proj.weight', 'model.layers.46.mlp.experts.122.gate_proj.weight', 'model.layers.46.mlp.experts.122.up_proj.weight', 'model.layers.46.mlp.experts.123.down_proj.weight', 'model.layers.46.mlp.experts.123.gate_proj.weight', 'model.layers.46.mlp.experts.123.up_proj.weight', 'model.layers.46.mlp.experts.124.down_proj.weight', 'model.layers.46.mlp.experts.124.gate_proj.weight', 'model.layers.46.mlp.experts.124.up_proj.weight', 'model.layers.46.mlp.experts.125.down_proj.weight', 'model.layers.46.mlp.experts.125.gate_proj.weight', 'model.layers.46.mlp.experts.125.up_proj.weight', 'model.layers.46.mlp.experts.126.down_proj.weight', 'model.layers.46.mlp.experts.126.gate_proj.weight', 'model.layers.46.mlp.experts.126.up_proj.weight', 'model.layers.46.mlp.experts.127.down_proj.weight', 'model.layers.46.mlp.experts.127.gate_proj.weight', 'model.layers.46.mlp.experts.127.up_proj.weight', 'model.layers.46.mlp.experts.13.down_proj.weight', 'model.layers.46.mlp.experts.13.gate_proj.weight', 'model.layers.46.mlp.experts.13.up_proj.weight', 'model.layers.46.mlp.experts.14.down_proj.weight', 'model.layers.46.mlp.experts.14.gate_proj.weight', 'model.layers.46.mlp.experts.14.up_proj.weight', 'model.layers.46.mlp.experts.15.down_proj.weight', 'model.layers.46.mlp.experts.15.gate_proj.weight', 'model.layers.46.mlp.experts.15.up_proj.weight', 'model.layers.46.mlp.experts.16.down_proj.weight', 'model.layers.46.mlp.experts.16.gate_proj.weight', 'model.layers.46.mlp.experts.16.up_proj.weight', 'model.layers.46.mlp.experts.17.down_proj.weight', 'model.layers.46.mlp.experts.17.gate_proj.weight', 'model.layers.46.mlp.experts.17.up_proj.weight', 'model.layers.46.mlp.experts.18.down_proj.weight', 'model.layers.46.mlp.experts.18.gate_proj.weight', 'model.layers.46.mlp.experts.18.up_proj.weight', 'model.layers.46.mlp.experts.19.down_proj.weight', 'model.layers.46.mlp.experts.19.gate_proj.weight', 'model.layers.46.mlp.experts.19.up_proj.weight', 'model.layers.46.mlp.experts.2.down_proj.weight', 'model.layers.46.mlp.experts.2.gate_proj.weight', 'model.layers.46.mlp.experts.2.up_proj.weight', 'model.layers.46.mlp.experts.20.down_proj.weight', 'model.layers.46.mlp.experts.20.gate_proj.weight', 'model.layers.46.mlp.experts.20.up_proj.weight', 'model.layers.46.mlp.experts.21.down_proj.weight', 'model.layers.46.mlp.experts.21.gate_proj.weight', 'model.layers.46.mlp.experts.21.up_proj.weight', 'model.layers.46.mlp.experts.22.down_proj.weight', 'model.layers.46.mlp.experts.22.gate_proj.weight', 'model.layers.46.mlp.experts.22.up_proj.weight', 'model.layers.46.mlp.experts.23.down_proj.weight', 'model.layers.46.mlp.experts.23.gate_proj.weight', 'model.layers.46.mlp.experts.23.up_proj.weight', 'model.layers.46.mlp.experts.24.down_proj.weight', 'model.layers.46.mlp.experts.24.gate_proj.weight', 'model.layers.46.mlp.experts.24.up_proj.weight', 'model.layers.46.mlp.experts.25.down_proj.weight', 'model.layers.46.mlp.experts.25.gate_proj.weight', 'model.layers.46.mlp.experts.25.up_proj.weight', 'model.layers.46.mlp.experts.26.down_proj.weight', 'model.layers.46.mlp.experts.26.gate_proj.weight', 'model.layers.46.mlp.experts.26.up_proj.weight', 'model.layers.46.mlp.experts.27.down_proj.weight', 'model.layers.46.mlp.experts.27.gate_proj.weight', 'model.layers.46.mlp.experts.27.up_proj.weight', 'model.layers.46.mlp.experts.28.down_proj.weight', 'model.layers.46.mlp.experts.28.gate_proj.weight', 'model.layers.46.mlp.experts.28.up_proj.weight', 'model.layers.46.mlp.experts.29.down_proj.weight', 'model.layers.46.mlp.experts.29.gate_proj.weight', 'model.layers.46.mlp.experts.29.up_proj.weight', 'model.layers.46.mlp.experts.3.down_proj.weight', 'model.layers.46.mlp.experts.3.gate_proj.weight', 'model.layers.46.mlp.experts.3.up_proj.weight', 'model.layers.46.mlp.experts.30.down_proj.weight', 'model.layers.46.mlp.experts.30.gate_proj.weight', 'model.layers.46.mlp.experts.30.up_proj.weight', 'model.layers.46.mlp.experts.31.down_proj.weight', 'model.layers.46.mlp.experts.31.gate_proj.weight', 'model.layers.46.mlp.experts.31.up_proj.weight', 'model.layers.46.mlp.experts.32.down_proj.weight', 'model.layers.46.mlp.experts.32.gate_proj.weight', 'model.layers.46.mlp.experts.32.up_proj.weight', 'model.layers.46.mlp.experts.33.down_proj.weight', 'model.layers.46.mlp.experts.33.gate_proj.weight', 'model.layers.46.mlp.experts.33.up_proj.weight', 'model.layers.46.mlp.experts.34.down_proj.weight', 'model.layers.46.mlp.experts.34.gate_proj.weight', 'model.layers.46.mlp.experts.34.up_proj.weight', 'model.layers.46.mlp.experts.35.down_proj.weight', 'model.layers.46.mlp.experts.35.gate_proj.weight', 'model.layers.46.mlp.experts.35.up_proj.weight', 'model.layers.46.mlp.experts.36.down_proj.weight', 'model.layers.46.mlp.experts.36.gate_proj.weight', 'model.layers.46.mlp.experts.36.up_proj.weight', 'model.layers.46.mlp.experts.37.down_proj.weight', 'model.layers.46.mlp.experts.37.gate_proj.weight', 'model.layers.46.mlp.experts.37.up_proj.weight', 'model.layers.46.mlp.experts.38.down_proj.weight', 'model.layers.46.mlp.experts.38.gate_proj.weight', 'model.layers.46.mlp.experts.38.up_proj.weight', 'model.layers.46.mlp.experts.39.down_proj.weight', 'model.layers.46.mlp.experts.39.gate_proj.weight', 'model.layers.46.mlp.experts.39.up_proj.weight', 'model.layers.46.mlp.experts.4.down_proj.weight', 'model.layers.46.mlp.experts.4.gate_proj.weight', 'model.layers.46.mlp.experts.4.up_proj.weight', 'model.layers.46.mlp.experts.40.down_proj.weight', 'model.layers.46.mlp.experts.40.gate_proj.weight', 'model.layers.46.mlp.experts.40.up_proj.weight', 'model.layers.46.mlp.experts.41.down_proj.weight', 'model.layers.46.mlp.experts.41.gate_proj.weight', 'model.layers.46.mlp.experts.41.up_proj.weight', 'model.layers.46.mlp.experts.42.down_proj.weight', 'model.layers.46.mlp.experts.42.gate_proj.weight', 'model.layers.46.mlp.experts.42.up_proj.weight', 'model.layers.46.mlp.experts.43.down_proj.weight', 'model.layers.46.mlp.experts.43.gate_proj.weight', 'model.layers.46.mlp.experts.43.up_proj.weight', 'model.layers.46.mlp.experts.44.down_proj.weight', 'model.layers.46.mlp.experts.44.gate_proj.weight', 'model.layers.46.mlp.experts.44.up_proj.weight', 'model.layers.46.mlp.experts.45.down_proj.weight', 'model.layers.46.mlp.experts.45.gate_proj.weight', 'model.layers.46.mlp.experts.45.up_proj.weight', 'model.layers.46.mlp.experts.46.down_proj.weight', 'model.layers.46.mlp.experts.46.gate_proj.weight', 'model.layers.46.mlp.experts.46.up_proj.weight', 'model.layers.46.mlp.experts.47.down_proj.weight', 'model.layers.46.mlp.experts.47.gate_proj.weight', 'model.layers.46.mlp.experts.47.up_proj.weight', 'model.layers.46.mlp.experts.48.down_proj.weight', 'model.layers.46.mlp.experts.48.gate_proj.weight', 'model.layers.46.mlp.experts.48.up_proj.weight', 'model.layers.46.mlp.experts.49.down_proj.weight', 'model.layers.46.mlp.experts.49.gate_proj.weight', 'model.layers.46.mlp.experts.49.up_proj.weight', 'model.layers.46.mlp.experts.5.down_proj.weight', 'model.layers.46.mlp.experts.5.gate_proj.weight', 'model.layers.46.mlp.experts.5.up_proj.weight', 'model.layers.46.mlp.experts.50.down_proj.weight', 'model.layers.46.mlp.experts.50.gate_proj.weight', 'model.layers.46.mlp.experts.50.up_proj.weight', 'model.layers.46.mlp.experts.51.down_proj.weight', 'model.layers.46.mlp.experts.51.gate_proj.weight', 'model.layers.46.mlp.experts.51.up_proj.weight', 'model.layers.46.mlp.experts.52.down_proj.weight', 'model.layers.46.mlp.experts.52.gate_proj.weight', 'model.layers.46.mlp.experts.52.up_proj.weight', 'model.layers.46.mlp.experts.53.down_proj.weight', 'model.layers.46.mlp.experts.53.gate_proj.weight', 'model.layers.46.mlp.experts.53.up_proj.weight', 'model.layers.46.mlp.experts.54.down_proj.weight', 'model.layers.46.mlp.experts.54.gate_proj.weight', 'model.layers.46.mlp.experts.54.up_proj.weight', 'model.layers.46.mlp.experts.55.down_proj.weight', 'model.layers.46.mlp.experts.55.gate_proj.weight', 'model.layers.46.mlp.experts.55.up_proj.weight', 'model.layers.46.mlp.experts.56.down_proj.weight', 'model.layers.46.mlp.experts.56.gate_proj.weight', 'model.layers.46.mlp.experts.56.up_proj.weight', 'model.layers.46.mlp.experts.57.down_proj.weight', 'model.layers.46.mlp.experts.57.gate_proj.weight', 'model.layers.46.mlp.experts.57.up_proj.weight', 'model.layers.46.mlp.experts.58.down_proj.weight', 'model.layers.46.mlp.experts.58.gate_proj.weight', 'model.layers.46.mlp.experts.58.up_proj.weight', 'model.layers.46.mlp.experts.59.down_proj.weight', 'model.layers.46.mlp.experts.59.gate_proj.weight', 'model.layers.46.mlp.experts.59.up_proj.weight', 'model.layers.46.mlp.experts.6.down_proj.weight', 'model.layers.46.mlp.experts.6.gate_proj.weight', 'model.layers.46.mlp.experts.6.up_proj.weight', 'model.layers.46.mlp.experts.60.down_proj.weight', 'model.layers.46.mlp.experts.60.gate_proj.weight', 'model.layers.46.mlp.experts.60.up_proj.weight', 'model.layers.46.mlp.experts.61.down_proj.weight', 'model.layers.46.mlp.experts.61.gate_proj.weight', 'model.layers.46.mlp.experts.61.up_proj.weight', 'model.layers.46.mlp.experts.62.down_proj.weight', 'model.layers.46.mlp.experts.62.gate_proj.weight', 'model.layers.46.mlp.experts.62.up_proj.weight', 'model.layers.46.mlp.experts.63.down_proj.weight', 'model.layers.46.mlp.experts.63.gate_proj.weight', 'model.layers.46.mlp.experts.63.up_proj.weight', 'model.layers.46.mlp.experts.64.down_proj.weight', 'model.layers.46.mlp.experts.64.gate_proj.weight', 'model.layers.46.mlp.experts.64.up_proj.weight', 'model.layers.46.mlp.experts.65.down_proj.weight', 'model.layers.46.mlp.experts.65.gate_proj.weight', 'model.layers.46.mlp.experts.65.up_proj.weight', 'model.layers.46.mlp.experts.66.down_proj.weight', 'model.layers.46.mlp.experts.66.gate_proj.weight', 'model.layers.46.mlp.experts.66.up_proj.weight', 'model.layers.46.mlp.experts.67.down_proj.weight', 'model.layers.46.mlp.experts.67.gate_proj.weight', 'model.layers.46.mlp.experts.67.up_proj.weight', 'model.layers.46.mlp.experts.68.down_proj.weight', 'model.layers.46.mlp.experts.68.gate_proj.weight', 'model.layers.46.mlp.experts.68.up_proj.weight', 'model.layers.46.mlp.experts.69.down_proj.weight', 'model.layers.46.mlp.experts.69.gate_proj.weight', 'model.layers.46.mlp.experts.69.up_proj.weight', 'model.layers.46.mlp.experts.7.down_proj.weight', 'model.layers.46.mlp.experts.7.gate_proj.weight', 'model.layers.46.mlp.experts.7.up_proj.weight', 'model.layers.46.mlp.experts.70.down_proj.weight', 'model.layers.46.mlp.experts.70.gate_proj.weight', 'model.layers.46.mlp.experts.70.up_proj.weight', 'model.layers.46.mlp.experts.71.down_proj.weight', 'model.layers.46.mlp.experts.71.gate_proj.weight', 'model.layers.46.mlp.experts.71.up_proj.weight', 'model.layers.46.mlp.experts.72.down_proj.weight', 'model.layers.46.mlp.experts.72.gate_proj.weight', 'model.layers.46.mlp.experts.72.up_proj.weight', 'model.layers.46.mlp.experts.73.down_proj.weight', 'model.layers.46.mlp.experts.73.gate_proj.weight', 'model.layers.46.mlp.experts.73.up_proj.weight', 'model.layers.46.mlp.experts.74.down_proj.weight', 'model.layers.46.mlp.experts.74.gate_proj.weight', 'model.layers.46.mlp.experts.74.up_proj.weight', 'model.layers.46.mlp.experts.75.down_proj.weight', 'model.layers.46.mlp.experts.75.gate_proj.weight', 'model.layers.46.mlp.experts.75.up_proj.weight', 'model.layers.46.mlp.experts.76.down_proj.weight', 'model.layers.46.mlp.experts.76.gate_proj.weight', 'model.layers.46.mlp.experts.76.up_proj.weight', 'model.layers.46.mlp.experts.77.down_proj.weight', 'model.layers.46.mlp.experts.77.gate_proj.weight', 'model.layers.46.mlp.experts.77.up_proj.weight', 'model.layers.46.mlp.experts.78.down_proj.weight', 'model.layers.46.mlp.experts.78.gate_proj.weight', 'model.layers.46.mlp.experts.78.up_proj.weight', 'model.layers.46.mlp.experts.79.down_proj.weight', 'model.layers.46.mlp.experts.79.gate_proj.weight', 'model.layers.46.mlp.experts.79.up_proj.weight', 'model.layers.46.mlp.experts.8.down_proj.weight', 'model.layers.46.mlp.experts.8.gate_proj.weight', 'model.layers.46.mlp.experts.8.up_proj.weight', 'model.layers.46.mlp.experts.80.down_proj.weight', 'model.layers.46.mlp.experts.80.gate_proj.weight', 'model.layers.46.mlp.experts.80.up_proj.weight', 'model.layers.46.mlp.experts.81.down_proj.weight', 'model.layers.46.mlp.experts.81.gate_proj.weight', 'model.layers.46.mlp.experts.81.up_proj.weight', 'model.layers.46.mlp.experts.82.down_proj.weight', 'model.layers.46.mlp.experts.82.gate_proj.weight', 'model.layers.46.mlp.experts.82.up_proj.weight', 'model.layers.46.mlp.experts.83.down_proj.weight', 'model.layers.46.mlp.experts.83.gate_proj.weight', 'model.layers.46.mlp.experts.83.up_proj.weight', 'model.layers.46.mlp.experts.84.down_proj.weight', 'model.layers.46.mlp.experts.84.gate_proj.weight', 'model.layers.46.mlp.experts.84.up_proj.weight', 'model.layers.46.mlp.experts.85.down_proj.weight', 'model.layers.46.mlp.experts.85.gate_proj.weight', 'model.layers.46.mlp.experts.85.up_proj.weight', 'model.layers.46.mlp.experts.86.down_proj.weight', 'model.layers.46.mlp.experts.86.gate_proj.weight', 'model.layers.46.mlp.experts.86.up_proj.weight', 'model.layers.46.mlp.experts.87.down_proj.weight', 'model.layers.46.mlp.experts.87.gate_proj.weight', 'model.layers.46.mlp.experts.87.up_proj.weight', 'model.layers.46.mlp.experts.88.down_proj.weight', 'model.layers.46.mlp.experts.88.gate_proj.weight', 'model.layers.46.mlp.experts.88.up_proj.weight', 'model.layers.46.mlp.experts.89.down_proj.weight', 'model.layers.46.mlp.experts.89.gate_proj.weight', 'model.layers.46.mlp.experts.89.up_proj.weight', 'model.layers.46.mlp.experts.9.down_proj.weight', 'model.layers.46.mlp.experts.9.gate_proj.weight', 'model.layers.46.mlp.experts.9.up_proj.weight', 'model.layers.46.mlp.experts.90.down_proj.weight', 'model.layers.46.mlp.experts.90.gate_proj.weight', 'model.layers.46.mlp.experts.90.up_proj.weight', 'model.layers.46.mlp.experts.91.down_proj.weight', 'model.layers.46.mlp.experts.91.gate_proj.weight', 'model.layers.46.mlp.experts.91.up_proj.weight', 'model.layers.46.mlp.experts.92.down_proj.weight', 'model.layers.46.mlp.experts.92.gate_proj.weight', 'model.layers.46.mlp.experts.92.up_proj.weight', 'model.layers.46.mlp.experts.93.down_proj.weight', 'model.layers.46.mlp.experts.93.gate_proj.weight', 'model.layers.46.mlp.experts.93.up_proj.weight', 'model.layers.46.mlp.experts.94.down_proj.weight', 'model.layers.46.mlp.experts.94.gate_proj.weight', 'model.layers.46.mlp.experts.94.up_proj.weight', 'model.layers.46.mlp.experts.95.down_proj.weight', 'model.layers.46.mlp.experts.95.gate_proj.weight', 'model.layers.46.mlp.experts.95.up_proj.weight', 'model.layers.46.mlp.experts.96.down_proj.weight', 'model.layers.46.mlp.experts.96.gate_proj.weight', 'model.layers.46.mlp.experts.96.up_proj.weight', 'model.layers.46.mlp.experts.97.down_proj.weight', 'model.layers.46.mlp.experts.97.gate_proj.weight', 'model.layers.46.mlp.experts.97.up_proj.weight', 'model.layers.46.mlp.experts.98.down_proj.weight', 'model.layers.46.mlp.experts.98.gate_proj.weight', 'model.layers.46.mlp.experts.98.up_proj.weight', 'model.layers.46.mlp.experts.99.down_proj.weight', 'model.layers.46.mlp.experts.99.gate_proj.weight', 'model.layers.46.mlp.experts.99.up_proj.weight', 'model.layers.46.mlp.gate.e_score_correction_bias', 'model.layers.46.mlp.gate.weight', 'model.layers.46.mlp.shared_experts.down_proj.weight', 'model.layers.46.mlp.shared_experts.gate_proj.weight', 'model.layers.46.mlp.shared_experts.up_proj.weight', 'model.layers.46.post_attention_layernorm.weight', 'model.layers.46.self_attn.k_proj.bias', 'model.layers.46.self_attn.k_proj.weight', 'model.layers.46.self_attn.o_proj.weight', 'model.layers.46.self_attn.q_proj.bias', 'model.layers.46.self_attn.q_proj.weight', 'model.layers.46.self_attn.v_proj.bias', 'model.layers.46.self_attn.v_proj.weight', 'model.layers.46.shared_head.head.weight', 'model.layers.46.shared_head.norm.weight']\n",
      "- This IS expected if you are initializing Glm4MoeForCausalLM from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing Glm4MoeForCausalLM from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
     ]
    }
   ],
   "source": [
    "from transformers import AutoModelForCausalLM, AutoTokenizer\n",
    "import torch\n",
    "from peft import PeftModel\n",
    "\n",
    "mode_path = '/model/ModelScope/ZhipuAI/GLM-4.5-Air'\n",
    "lora_path = '/workspace/output/glm45_air_lora/checkpoint-20' # 这里改称你的 lora 输出对应 checkpoint 地址\n",
    "\n",
    "# 加载tokenizer\n",
    "tokenizer = AutoTokenizer.from_pretrained(mode_path)\n",
    "\n",
    "# 加载模型\n",
    "model = AutoModelForCausalLM.from_pretrained(mode_path, device_map=\"auto\",torch_dtype=torch.bfloat16, trust_remote_code=True)\n",
    "\n",
    "# 加载lora权重\n",
    "model = PeftModel.from_pretrained(model, model_id=lora_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2dff0799",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/miniconda3/envs/py310/lib/python3.10/site-packages/transformers/generation/utils.py:2505: UserWarning: You are calling .generate() with the `input_ids` being on a device type different than your model's device. `input_ids` is on cpu, whereas the model is on cuda. You may experience unexpected behaviors or slower generation. Please make sure that you have put `input_ids` to the correct device by calling for example input_ids = input_ids.to('cuda') before running `.generate()`.\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "prompt = \"你是谁？\"\n",
    "inputs = tokenizer.apply_chat_template(\n",
    "                                    [{\"role\": \"user\", \"content\": \"假设你是皇帝身边的女人--甄嬛。\"},{\"role\": \"user\", \"content\": prompt}],\n",
    "                                    add_generation_prompt=True,\n",
    "                                    tokenize=True,\n",
    "                                    return_tensors=\"pt\",\n",
    "                                    return_dict=True,\n",
    "                                    enable_thinking=False\n",
    "                                )\n",
    "\n",
    "inputs = {\n",
    "    \"input_ids\": inputs[\"input_ids\"],\n",
    "    \"attention_mask\": inputs[\"attention_mask\"]\n",
    "}\n",
    "\n",
    "gen_kwargs = {\"max_length\": 2500, \"do_sample\": True, \"top_k\": 1}\n",
    "with torch.no_grad():\n",
    "    outputs = model.generate(**inputs, **gen_kwargs)\n",
    "    outputs = outputs[:, inputs['input_ids'].shape[1]:]\n",
    "    print(tokenizer.decode(outputs[0], skip_special_tokens=True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c7ba8ac0",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "py310",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
